import os,csv,sys
from datetime import datetime
import time

distinctVal=[]
colNames=[]
#colAlphaNum contains 'a' if the attribute is alphanumeric and 'n' if it is numeric
colAlphaNum=['n']*273

tstart=datetime.now()
os.chdir('C:\Hearst')
newModel=open('newModeling.csv')
validation=open('validation.csv')
distinctValFile=open('distinctCities_new.csv','w')
csvfile=csv.reader(newModel,delimiter=',',quotechar='"')
validationfile=csv.reader(validation,delimiter=',',quotechar='"')

for colname in csvfile.next():
    colNames.append(colname)
    distinctVal.append({})

cityIdx=colNames.index('CITY')
stateIdx=colNames.index('STATE_PROVINCE')


for row in csvfile:
    if len(row)>273:
        print row
        sys.exit(2)
    for ctr,val in enumerate(row):
        if ctr==cityIdx:
            distinctVal[ctr][val+','+row[stateIdx]]=1
        else:
            continue
        
#ignore the first line containing headers in the validation file
validationfile.next()

for row in validationfile:
    if len(row)>273:
        print row
        sys.exit(2)
    for ctr,val in enumerate(row):
        if ctr==cityIdx:
            distinctVal[ctr][val+','+row[stateIdx]]=1
        else:
            continue

#write the distinct values for attributes with distinct value count < 1000
for item in distinctVal[cityIdx].keys():
    distinctValFile.write(item+'\n')

distinctValFile.close()

